{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from selenium import webdriver\n",
    "from selenium.webdriver import ActionChains\n",
    "import time\n",
    "import random\n",
    "import time\n",
    "import requests\n",
    "import re\n",
    "import json\n",
    "import pickle\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# def get_proxy_ip():\n",
    "#     try:\n",
    "#         zhima_ip = r'http://webapi.http.zhimacangku.com/getip?num=1&type=1&pro=&city=0&yys=0&port=11&pack=34560&ts=0&ys=0&cs=0&lb=1&sb=0&pb=4&mr=1&regions='\n",
    "#         proxy = requests.get(zhima_ip).text[:-2]\n",
    "#         time.sleep(1.5)\n",
    "#         prox = {\n",
    "#             'https': 'https://{}'.format(proxy),\n",
    "#             'http': 'http://{}'.format(proxy)}\n",
    "#         res = requests.get('http://2018.ip138.com/ic.asp', proxies=prox, timeout=10)\n",
    "#         if res.status_code == 200:\n",
    "#             return proxy.split(':')\n",
    "#         else:\n",
    "#             return get_proxy_ip()\n",
    "#     except:\n",
    "#         return get_proxy_ip()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# proxy_url, port = get_proxy_ip()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "metadata": {},
   "outputs": [],
   "source": [
    "user_agent = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/44.3538.102 Safari/537.36'\n",
    "# user_agent = \"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.110 Safari/537.36\"\n",
    "# user_agent = 'Mozilla/5.0 (Windows NT 10.0; WOW64; Trident/7.0; .NET4.0C; .NET4.0E; .NET CLR 2.0.50727; .NET CLR 3.0.30729; .NET CLR 3.5.30729; InfoPath.3; rv:11.0) like Gecko'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "metadata": {},
   "outputs": [],
   "source": [
    "fp = webdriver.FirefoxProfile()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "metadata": {},
   "outputs": [],
   "source": [
    "fp.set_preference(\"general.useragent.override\", user_agent)\n",
    "# fp.set_preference('network.proxy.type', 1)\n",
    "# fp.set_preference('network.proxy.http', proxy_url)\n",
    "# fp.set_preference('network.proxy.http_port', int(port))\n",
    "# fp.set_preference('network.proxy.ssl', proxy_url)\n",
    "# fp.set_preference('network.proxy.ssl_port', int(port))\n",
    "fp.update_preferences()\n",
    "driver = webdriver.Firefox(firefox_profile=fp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {},
   "outputs": [],
   "source": [
    "# driver.get('http://www.useragentstring.com/')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "metadata": {},
   "outputs": [],
   "source": [
    "def set_navigator():\n",
    "    script_webdriver = '''Object.defineProperties(navigator,{ webdriver:{ get:() => undefined } })'''\n",
    "    script_language = ''' Object.defineProperty(navigator, 'languages', { get: () => [ \"zh-CN\", \"zh\", \"zh-TW\", \"zh-HK\", \"en-US\", \"en\" ], });'''\n",
    "    driver.execute_script(script_webdriver)\n",
    "    driver.execute_script(script_language)\n",
    "    driver.execute_script('''() =>{ Object.defineProperties(navigator,{ webdriver:{ get: () => undefined } }) }''')  # 以下为插入中间js，将淘宝会为了检测浏览器而调用的js修改其结果。\n",
    "    driver.execute_script('''() =>{ window.navigator.chrome = { runtime: {},  }; }''')\n",
    "    driver.execute_script('''() =>{ Object.defineProperty(navigator, 'languages', { get: () => ['en-US', 'en'] }); }''')\n",
    "    driver.execute_script('''() =>{ Object.defineProperty(navigator, 'plugins', { get: () => [1, 2, 3, 4, 5,6], }); }''')\n",
    "    print('navigator set successfully')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "navigator set successfully\n"
     ]
    }
   ],
   "source": [
    "set_navigator()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "metadata": {},
   "outputs": [],
   "source": [
    "taobao_home = r'https://login.taobao.com/member/login.jhtml?style=mini&css_style=b2b&from=b2b&full_redirect=true'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "metadata": {},
   "outputs": [],
   "source": [
    "driver.get(taobao_home)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "metadata": {},
   "outputs": [],
   "source": [
    "driver.find_element_by_css_selector('#TPL_username_1').send_keys('17621780176')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 124,
   "metadata": {},
   "outputs": [],
   "source": [
    "driver.find_element_by_css_selector('#TPL_password_1').send_keys('www.,194928.,com')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 125,
   "metadata": {},
   "outputs": [],
   "source": [
    "driver.find_element_by_css_selector('#J_SubmitStatic').click()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 126,
   "metadata": {},
   "outputs": [],
   "source": [
    "scrorelump = driver.find_elements_by_xpath(\"//*[@id='nc_1_n1t']/span\")\n",
    "if scrorelump:\n",
    "    driver.find_element_by_css_selector('#TPL_password_1').send_keys('www.,194928.,com')\n",
    "    action = ActionChains(driver)\n",
    "    source=driver.find_element_by_xpath(\"//*[@id='nc_1_n1t']/span\")#需要滑动的元素\n",
    "    action.click_and_hold(source).perform()\n",
    "    time.sleep(0.5)\n",
    "    action.move_by_offset(250,0)#需要滑动的坐标\n",
    "    action.release().perform() #释放鼠标\n",
    "    time.sleep(random.random()*5)\n",
    "else:\n",
    "    driver.find_element_by_css_selector('#TPL_username_1').clear()\n",
    "    driver.find_element_by_css_selector('#TPL_username_1').send_keys('17621780176')\n",
    "    driver.find_element_by_css_selector('#TPL_password_1').send_keys('www.,194928.,com')\n",
    "driver.find_element_by_css_selector('#J_SubmitStatic').click()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 127,
   "metadata": {},
   "outputs": [],
   "source": [
    "driver.get('https://only.m.tmall.com/shop/shop_auction_search.htm?sort=default')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "driver.fin"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "navigator set successfully\n"
     ]
    }
   ],
   "source": [
    "driver.refresh()\n",
    "set_navigator()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "# driver.firefox_profile.update_preferences()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# driver.firefox_profile.default_preferences['network.proxy.type'] = 1\n",
    "# driver.firefox_profile.default_preferences['network.proxy.http'] = proxy_url\n",
    "# driver.firefox_profile.default_preferences['network.proxy.http_port'] = int(port)\n",
    "# driver.firefox_profile.default_preferences['network.proxy.ssl'] = proxy_url\n",
    "# driver.firefox_profile.default_preferences['network.proxy.ssl_port'] = port\n",
    "# driver.firefox_profile.update_preferences()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.77 Safari/537.36'"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.77 Safari/537.36'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "ename": "NoSuchElementException",
     "evalue": "Message: Unable to locate element: #sufei-dialog-content\n",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mNoSuchElementException\u001b[0m                    Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-11-818e99361e5a>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[0miframe\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mdriver\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfind_element_by_css_selector\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'#sufei-dialog-content'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      2\u001b[0m \u001b[0mdriver\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mswitch_to\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mframe\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0miframe\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\ProgramData\\Anaconda3\\lib\\site-packages\\selenium\\webdriver\\remote\\webdriver.py\u001b[0m in \u001b[0;36mfind_element_by_css_selector\u001b[1;34m(self, css_selector)\u001b[0m\n\u001b[0;32m    596\u001b[0m             \u001b[0melement\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mdriver\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfind_element_by_css_selector\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'#foo'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    597\u001b[0m         \"\"\"\n\u001b[1;32m--> 598\u001b[1;33m         \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfind_element\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mby\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mBy\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mCSS_SELECTOR\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mvalue\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mcss_selector\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    599\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    600\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0mfind_elements_by_css_selector\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcss_selector\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\ProgramData\\Anaconda3\\lib\\site-packages\\selenium\\webdriver\\remote\\webdriver.py\u001b[0m in \u001b[0;36mfind_element\u001b[1;34m(self, by, value)\u001b[0m\n\u001b[0;32m    976\u001b[0m         return self.execute(Command.FIND_ELEMENT, {\n\u001b[0;32m    977\u001b[0m             \u001b[1;34m'using'\u001b[0m\u001b[1;33m:\u001b[0m \u001b[0mby\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 978\u001b[1;33m             'value': value})['value']\n\u001b[0m\u001b[0;32m    979\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    980\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0mfind_elements\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mby\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mBy\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mID\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mvalue\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mNone\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\ProgramData\\Anaconda3\\lib\\site-packages\\selenium\\webdriver\\remote\\webdriver.py\u001b[0m in \u001b[0;36mexecute\u001b[1;34m(self, driver_command, params)\u001b[0m\n\u001b[0;32m    319\u001b[0m         \u001b[0mresponse\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcommand_executor\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mexecute\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdriver_command\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mparams\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    320\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mresponse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 321\u001b[1;33m             \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0merror_handler\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcheck_response\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mresponse\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    322\u001b[0m             response['value'] = self._unwrap_value(\n\u001b[0;32m    323\u001b[0m                 response.get('value', None))\n",
      "\u001b[1;32mC:\\ProgramData\\Anaconda3\\lib\\site-packages\\selenium\\webdriver\\remote\\errorhandler.py\u001b[0m in \u001b[0;36mcheck_response\u001b[1;34m(self, response)\u001b[0m\n\u001b[0;32m    240\u001b[0m                 \u001b[0malert_text\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mvalue\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'alert'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mget\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'text'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    241\u001b[0m             \u001b[1;32mraise\u001b[0m \u001b[0mexception_class\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mmessage\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mscreen\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mstacktrace\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0malert_text\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 242\u001b[1;33m         \u001b[1;32mraise\u001b[0m \u001b[0mexception_class\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mmessage\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mscreen\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mstacktrace\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    243\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    244\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0m_value_or_default\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mobj\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mkey\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdefault\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mNoSuchElementException\u001b[0m: Message: Unable to locate element: #sufei-dialog-content\n"
     ]
    }
   ],
   "source": [
    "iframe = driver.find_element_by_css_selector('#sufei-dialog-content')\n",
    "driver.switch_to.frame(iframe)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "driver.find_element_by_css_selector('#TPL_username_1').send_keys('17621780176')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "driver.find_element_by_css_selector('#TPL_password_1').send_keys('www.,194928.,com')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "driver.find_element_by_css_selector('#J_SubmitStatic').click()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "driver.find_element_by_css_selector('#TPL_password_1').send_keys('www.,194928.,com')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 214,
   "metadata": {},
   "outputs": [],
   "source": [
    "action = ActionChains(driver)\n",
    "source=driver.find_element_by_xpath(\"//*[@id='nc_1_n1t']/span\")\n",
    "#需要滑动的元素\n",
    "action.click_and_hold(source).perform()\n",
    "time.sleep(0.5)\n",
    "action.move_by_offset(206,0)#需要滑动的坐标\n",
    "time.sleep(1.2)\n",
    "action.release().perform() #释放鼠标"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [],
   "source": [
    "# script_refresh = 'javascript:noCaptcha.reset(1)'\n",
    "# driver.execute_script(script_refresh)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "driver.find_element_by_css_selector('#J_SubmitStatic').click()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 缺少处理如何滑动解锁的步骤"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 128,
   "metadata": {},
   "outputs": [],
   "source": [
    "driver.get('https://semir.m.tmall.com/shop/shop_auction_search.htm')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "navigator set successfully\n"
     ]
    }
   ],
   "source": [
    "driver.refresh()\n",
    "set_navigator()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'滑动解锁完成'"
      ]
     },
     "execution_count": 59,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# iframe2 = driver.find_element_by_css_selector('#sufei-dialog-content')\n",
    "def scroll_lump_code():\n",
    "    try:\n",
    "        iframe = driver.find_element_by_css_selector('#sufei-dialog-content')\n",
    "        driver.switch_to.frame(iframe)\n",
    "    except:\n",
    "        print('no need switch to iframe')\n",
    "    try:\n",
    "        action = ActionChains(driver)\n",
    "        source=driver.find_element_by_xpath(\"//*[@id='nc_1_n1t']/span\")#需要滑动的元素\n",
    "        action.click_and_hold(source).perform()\n",
    "        time.sleep(0.1)\n",
    "        action.move_by_offset(258,0)#需要滑动的坐标\n",
    "        time.sleep(random.random()*5)\n",
    "        action.release().perform() #释放鼠标\n",
    "    except:\n",
    "        pass\n",
    "    return '滑动解锁完成'\n",
    "scroll_lump_code()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {},
   "outputs": [],
   "source": [
    "driver.get('https://only.m.tmall.com/shop/shop_auction_search.htm?suid=356060330&sort=default')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {},
   "outputs": [],
   "source": [
    "scrollHeight = driver.execute_script('''return document.body.scrollHeight''')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<selenium.webdriver.firefox.webelement.FirefoxWebElement (session=\"1e7c6a99-efc8-49c8-a09f-02fbafacb2c7\", element=\"c7c845a8-68c1-44e6-b817-7bade2442b1d\")>]"
      ]
     },
     "execution_count": 102,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "driver.find_elements_by_id('footer')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "def scroll_to_bottom()\n",
    "    for i in range(30):\n",
    "        time.sleep(1)\n",
    "        driver.execute_script(\"window.scrollTo(0, document.body.scrollHeight)\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "# for i in range(10):\n",
    "#     a = i * 10000\n",
    "#     b = (i+1) * 10000\n",
    "#     script = \"window.scrollTo({},{})\".format(a, b)\n",
    "#     driver.execute_script(script)\n",
    "#     time.sleep(random.randint(1, 3) + 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "def func_totalpage():\n",
    "    firt_page_url = 'https://{}.m.tmall.com/shop/shop_auction_search.do?ajson=1&_tm_source=tmallsearch&sort=s&p=1&page_size=12&from=h5&shop_id=57303596&callback=jsonp_{}'.format(shop_name, num)\n",
    "    driver.get(firt_page_url)\n",
    "    html = driver.page_source\n",
    "    try:\n",
    "        infos = re.findall(r'\\(({.*})\\)', html)[0]\n",
    "        infos = json.loads(infos)\n",
    "        totalpage = int(infos.get('total_page'))\n",
    "        return int(totalpage) #*1.04\n",
    "    except:\n",
    "        raise IndexError"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_totalpage(shop_name):\n",
    "    try:\n",
    "        totalpage = func_totalpage()\n",
    "    except IndexError:\n",
    "        driver.get('https://{}.m.tmall.com/shop/shop_auction_search.htm?sort=default'.format(shop_name))\n",
    "        time.sleep(random.random()*5 + 1)\n",
    "        hand = input('\\n总页数，请手动滑动解锁\\n')\n",
    "        if hand == 'b':\n",
    "            raise ValueError\n",
    "        else:\n",
    "            pass\n",
    "#         print(outside_code())\n",
    "        totalpage = func_totalpage()\n",
    "    return totalpage"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "def try_totalpage():\n",
    "    try:\n",
    "        totalpage = get_totalpage(shop_name)\n",
    "        return totalpage\n",
    "    except IndexError:\n",
    "        return try_totalpage()\n",
    "    except ValueError:\n",
    "        return 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_one_page(shop_name, i, num):\n",
    "    print('---第',i,'页正在加载')\n",
    "    url = 'https://{}.m.tmall.com/shop/shop_auction_search.do?ajson=1&_tm_source=tmallsearch&sort=s&p={}&page_size=12&from=h5&callback=jsonp_{}'.format(shop_name, i, num)\n",
    "    #https://handuyishe.m.tmall.com/shop/shop_auction_search.do?sort=s&p=1&page_size=12&from=h5&ajson=1&_tm_source=tmallsearch&callback=jsonp_42750715\n",
    "    driver.get(url)\n",
    "    time.sleep(random.random() + 1)\n",
    "    html = driver.page_source\n",
    "    if re.findall(r'\\(({.*})\\)', html):\n",
    "        infos = re.findall(r'\\(({.*})\\)', html)[0]\n",
    "        infos = json.loads(infos)\n",
    "        products = infos.get('items')\n",
    "        if products:\n",
    "            for one_item in products:\n",
    "                one_item['store_name'] = shop_name\n",
    "                one_item['page'] = str(i)\n",
    "        else:\n",
    "            raise IndexError\n",
    "    else:\n",
    "        raise IndexError\n",
    "    return products"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "# store_name = input('请输入店铺名： ')\n",
    "def get_one_store(brand_name, shop_url, z):\n",
    "    shop_name = shop_url[8:-12]\n",
    "    all_page = []\n",
    "    for i in range(z, totalpage+1):\n",
    "        print('当前页面是: 第', i, '页')\n",
    "        try:\n",
    "            products = get_one_page(shop_name, i, num) #获取每页的数据\n",
    "            with open('d:\\\\test\\\\{}_1130'.format(brand_name), 'a+', encoding='utf-8', errors='ignore') as f:\n",
    "                f.write(str(all_page))\n",
    "            df = pd.DataFrame(products)\n",
    "            col = df.columns\n",
    "            df.to_csv('D:\\\\Jupyter\\\\all_brand\\\\{}_1130.csv'.format(brand_name), mode='a+', header=False, columns=col)\n",
    "            print('---第 {} 页加载完成\\n'.format(i))\n",
    "        except (TypeError, IndexError):\n",
    "            driver.get('{}/shop/shop_auction_search.htm?sort=default'.format(shop_url))\n",
    "            time.sleep(random.random()*5 + 1)\n",
    "            hand = input('\\n单页，请手动滑动解锁\\n')\n",
    "            print(hand)\n",
    "            if hand == '1':\n",
    "                break\n",
    "            else:\n",
    "                pass\n",
    "    return '------{} finished!'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "def try_one_store():\n",
    "    try:\n",
    "        print(get_one_store(brand_name, shop_url, z))\n",
    "    except IndexError:\n",
    "         if 'b' == input('need break: '):\n",
    "            return None\n",
    "         else:\n",
    "            return try_one_store()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_one_url():\n",
    "    \"\"\"从文件获取需要爬取的url链接\"\"\"\n",
    "    url_file = r\"D:\\Python\\TmallCrawler\\file\\Tamll_shop_all\"\n",
    "    with open(url_file, 'r', encoding='utf-8') as f:\n",
    "        csv_reader = f.read().split('\\n')\n",
    "        for row in csv_reader:\n",
    "            yield row.split('\\t')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "# driver = webdriver.Firefox(firefox_profile=fp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "navigator set successfully\n"
     ]
    }
   ],
   "source": [
    "set_navigator()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "韩都衣舍  is finished\n",
      "--------: \n",
      "韩都衣舍 is beginning\n",
      "\n",
      "总页数，请手动滑动解锁\n",
      "\n",
      "韩都衣舍 have 141 page items\n",
      "起始页数： 122\n",
      "当前页面是: 第 122 页\n",
      "---第 122 页正在加载\n",
      "---第 122 页加载完成\n",
      "\n",
      "当前页面是: 第 123 页\n",
      "---第 123 页正在加载\n",
      "---第 123 页加载完成\n",
      "\n",
      "当前页面是: 第 124 页\n",
      "---第 124 页正在加载\n",
      "---第 124 页加载完成\n",
      "\n",
      "当前页面是: 第 125 页\n",
      "---第 125 页正在加载\n",
      "---第 125 页加载完成\n",
      "\n",
      "当前页面是: 第 126 页\n",
      "---第 126 页正在加载\n",
      "---第 126 页加载完成\n",
      "\n",
      "当前页面是: 第 127 页\n",
      "---第 127 页正在加载\n",
      "---第 127 页加载完成\n",
      "\n",
      "当前页面是: 第 128 页\n",
      "---第 128 页正在加载\n",
      "---第 128 页加载完成\n",
      "\n",
      "当前页面是: 第 129 页\n",
      "---第 129 页正在加载\n",
      "---第 129 页加载完成\n",
      "\n",
      "当前页面是: 第 130 页\n",
      "---第 130 页正在加载\n",
      "---第 130 页加载完成\n",
      "\n",
      "当前页面是: 第 131 页\n",
      "---第 131 页正在加载\n",
      "---第 131 页加载完成\n",
      "\n",
      "当前页面是: 第 132 页\n",
      "---第 132 页正在加载\n",
      "---第 132 页加载完成\n",
      "\n",
      "当前页面是: 第 133 页\n",
      "---第 133 页正在加载\n",
      "---第 133 页加载完成\n",
      "\n",
      "当前页面是: 第 134 页\n",
      "---第 134 页正在加载\n",
      "---第 134 页加载完成\n",
      "\n",
      "当前页面是: 第 135 页\n",
      "---第 135 页正在加载\n",
      "---第 135 页加载完成\n",
      "\n",
      "当前页面是: 第 136 页\n",
      "---第 136 页正在加载\n",
      "---第 136 页加载完成\n",
      "\n",
      "当前页面是: 第 137 页\n",
      "---第 137 页正在加载\n",
      "---第 137 页加载完成\n",
      "\n",
      "当前页面是: 第 138 页\n",
      "---第 138 页正在加载\n",
      "---第 138 页加载完成\n",
      "\n",
      "当前页面是: 第 139 页\n",
      "---第 139 页正在加载\n",
      "---第 139 页加载完成\n",
      "\n",
      "当前页面是: 第 140 页\n",
      "---第 140 页正在加载\n",
      "---第 140 页加载完成\n",
      "\n",
      "当前页面是: 第 141 页\n",
      "---第 141 页正在加载\n",
      "---第 141 页加载完成\n",
      "\n",
      "------{} finished!\n",
      "None\n",
      "优衣库  is finished\n",
      "--------: \n",
      "优衣库 is beginning\n",
      "\n",
      "总页数，请手动滑动解锁\n",
      "\n",
      "优衣库 have 81 page items\n",
      "起始页数： 1\n",
      "当前页面是: 第 1 页\n",
      "---第 1 页正在加载\n",
      "\n",
      "单页，请手动滑动解锁\n",
      "\n",
      "\n",
      "当前页面是: 第 2 页\n",
      "---第 2 页正在加载\n",
      "---第 2 页加载完成\n",
      "\n",
      "当前页面是: 第 3 页\n",
      "---第 3 页正在加载\n",
      "---第 3 页加载完成\n",
      "\n",
      "当前页面是: 第 4 页\n",
      "---第 4 页正在加载\n",
      "---第 4 页加载完成\n",
      "\n",
      "当前页面是: 第 5 页\n",
      "---第 5 页正在加载\n",
      "---第 5 页加载完成\n",
      "\n",
      "当前页面是: 第 6 页\n",
      "---第 6 页正在加载\n",
      "---第 6 页加载完成\n",
      "\n",
      "当前页面是: 第 7 页\n",
      "---第 7 页正在加载\n",
      "---第 7 页加载完成\n",
      "\n",
      "当前页面是: 第 8 页\n",
      "---第 8 页正在加载\n",
      "---第 8 页加载完成\n",
      "\n",
      "当前页面是: 第 9 页\n",
      "---第 9 页正在加载\n",
      "---第 9 页加载完成\n",
      "\n",
      "当前页面是: 第 10 页\n",
      "---第 10 页正在加载\n",
      "---第 10 页加载完成\n",
      "\n",
      "当前页面是: 第 11 页\n",
      "---第 11 页正在加载\n",
      "---第 11 页加载完成\n",
      "\n",
      "当前页面是: 第 12 页\n",
      "---第 12 页正在加载\n",
      "---第 12 页加载完成\n",
      "\n",
      "当前页面是: 第 13 页\n",
      "---第 13 页正在加载\n",
      "---第 13 页加载完成\n",
      "\n",
      "当前页面是: 第 14 页\n",
      "---第 14 页正在加载\n",
      "---第 14 页加载完成\n",
      "\n",
      "当前页面是: 第 15 页\n",
      "---第 15 页正在加载\n",
      "---第 15 页加载完成\n",
      "\n",
      "当前页面是: 第 16 页\n",
      "---第 16 页正在加载\n",
      "---第 16 页加载完成\n",
      "\n",
      "当前页面是: 第 17 页\n",
      "---第 17 页正在加载\n",
      "---第 17 页加载完成\n",
      "\n",
      "当前页面是: 第 18 页\n",
      "---第 18 页正在加载\n",
      "---第 18 页加载完成\n",
      "\n",
      "当前页面是: 第 19 页\n",
      "---第 19 页正在加载\n",
      "---第 19 页加载完成\n",
      "\n",
      "当前页面是: 第 20 页\n",
      "---第 20 页正在加载\n",
      "---第 20 页加载完成\n",
      "\n",
      "当前页面是: 第 21 页\n",
      "---第 21 页正在加载\n",
      "---第 21 页加载完成\n",
      "\n",
      "当前页面是: 第 22 页\n",
      "---第 22 页正在加载\n",
      "---第 22 页加载完成\n",
      "\n",
      "当前页面是: 第 23 页\n",
      "---第 23 页正在加载\n",
      "---第 23 页加载完成\n",
      "\n",
      "当前页面是: 第 24 页\n",
      "---第 24 页正在加载\n",
      "\n",
      "单页，请手动滑动解锁\n",
      "\n",
      "\n",
      "当前页面是: 第 25 页\n",
      "---第 25 页正在加载\n",
      "---第 25 页加载完成\n",
      "\n",
      "当前页面是: 第 26 页\n",
      "---第 26 页正在加载\n"
     ]
    }
   ],
   "source": [
    "for i in get_one_url():\n",
    "    brand_name, shop_url = i\n",
    "    print(brand_name, ' is finished')\n",
    "    a = input('--------: ')\n",
    "    if a == 'y':\n",
    "        pass\n",
    "    else:\n",
    "        shop_name = shop_url[8:-12]\n",
    "        print('{} is beginning'.format(brand_name))\n",
    "        num = random.randint(10127482, 75727482)\n",
    "        totalpage = try_totalpage()\n",
    "        print('{} have {} page items'.format(brand_name, totalpage))\n",
    "        z = int(input(\"起始页数： \"))\n",
    "        print(try_one_store())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# shop_name\n",
    "driver.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "# df = pd.DataFrame(all_page)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# with open('test', 'wb') as f:\n",
    "#     f.write(pickle.dumps(all_page))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "dff = pd.DataFrame([1,2,3,4,5])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "ename": "AttributeError",
     "evalue": "module 'pandas' has no attribute 'to_csv'",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mAttributeError\u001b[0m                            Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-46-1e6d63ba9ef5>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[0mpd\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mto_csv\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;31mAttributeError\u001b[0m: module 'pandas' has no attribute 'to_csv'"
     ]
    }
   ],
   "source": [
    "dff.to_csv('')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
